﻿using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
/*
 * convert raw corpus to crfpp training data format, a example as follows:
 * raw corpus format:
 * 付仲恺[PER] 和[NOR] 张晓晨[PER] 一起[NOR] 在家[NOR] 看[NOR] 非诚勿扰[VDO_TVSHOW] 。[NOR]
 * crfpp training data format
 * 付 B_PER
 * 仲 M_PER
 * 恺 E_PER
 * 和 S
 * 张 B_PER
 * 晓 M_PER
 * 晨 E_PER
 * 一 B
 * 起 E
 * 在 B
 * 家 E
 * 看 S
 * 非 B_VDO_TVSHOW
 * 诚 M_VDO_TVSHOW
 * 勿 M_VDO_TVSHOW
 * 扰 E_VDO_TVSHOW
 * 。 S
*/
namespace corpus2tag
{
    class Program
    {
        static void Main(string[] args)
        {
            if (args.Length != 3)
            {
                Console.WriteLine("corpus2tag [raw corpus filename] [crf term tag filename] [type mapping filename]");
                return;
            }

            Dictionary<string, int> tag2num = new Dictionary<string, int>();
            Dictionary<string, string> t2t = new Dictionary<string, string>();
            StreamReader sr = new StreamReader(args[0]);
            StreamWriter sw = new StreamWriter(args[1]);
            StreamReader swmap = new StreamReader(args[2]);

            while (swmap.EndOfStream == false)
            {
                string strLine = swmap.ReadLine().Trim();
                string [] items = strLine.Split(new char[]{'\t'});

                if (items.Length != 2)
                {
                    Console.WriteLine("{0} is incorrect format", strLine);
                    continue;
                }

                if (t2t.ContainsKey(items[0]) == false)
                {
                    t2t.Add(items[0], items[1]);
                }
                else
                {
                    Console.WriteLine("{0} is duplicated", items[0]);
                }
            }
            swmap.Close();


            while (sr.EndOfStream == false)
            {
                string strLine = sr.ReadLine();
                if (strLine.Length == 0)
                {
                    continue;
                }

                string[] items = strLine.Split();
                string term, tag;

                foreach (string item in items)
                {
                    if (item[item.Length - 1] != ']')
                    {
                        Console.WriteLine("{0} is invalidated format", strLine);
                        break;
                    }

                    int i;
                    term = "";
                    tag = "";
                    for (i = item.Length - 1; i >= 0; i--)
                    {
                        if (item[i] == '[')
                        {
                            tag = item.Substring(i + 1, item.Length - i - 2);
                            term = item.Substring(0, i).ToLower();
                            break;
                        }
                    }

                    if (term.Length == 0 || tag.Length == 0)
                    {
                        Console.WriteLine("Invalidate line: {0}", strLine);
                    }

                    if (t2t.ContainsKey(tag) == true)
                    {
                        tag = t2t[tag];
                    }

                    if (tag == "NOR")
                    {
                        tag = "";
                    }

                    if (term.Length == 1)
                    {
                        if (tag.Length == 0)
                        {
                            tag = "S";
                        }
                        else
                        {
                            tag = "S_" + tag;
                        }

                        if (tag2num.ContainsKey(tag) == false)
                        {
                            tag2num.Add(tag, 1);
                        }
                        else
                        {
                            tag2num[tag]++;
                        }
                        if (tag.Trim().Length == 0)
                        {
                            Console.WriteLine("ERROR: {0}", strLine);
                        }

                        sw.WriteLine("{0}\t{1}", term, tag);
                        continue;
                    }

                    for (i = 0; i < term.Length; i++)
                    {
                        string tag2;
                        if (tag.Length > 0)
                        {
                            if (i == 0)
                            {
                                tag2 = "B_" + tag;
                            }
                            else if (i == term.Length - 1)
                            {
                                tag2 = "E_" + tag;
                            }
                            //else if (i == term.Length - 2)
                            //{
                            //    tag2 = "E2_" + tag;
                            //}
                            //else if (i == term.Length - 3)
                            //{
                            //    tag2 = "E3_" + tag;
                            //}
                            //else if (i == 1)
                            //{
                            //    tag2 = "B2_" + tag;
                            //}
                            //else if (i == 2)
                            //{
                            //    tag2 = "B3_" + tag;
                            //}
                            else
                            {
                                tag2 = "M_" + tag;
                            }
                        }
                        else
                        {
                            if (i == 0)
                            {
                                tag2 = "B";
                            }
                            else if (i == term.Length - 1)
                            {
                                tag2 = "E";
                            }
                            //else if (i == term.Length - 2)
                            //{
                            //    tag2 = "E2";
                            //}
                            //else if (i == term.Length - 3)
                            //{
                            //    tag2 = "E3";
                            //}
                            //else if (i == 1)
                            //{
                            //    tag2 = "B2";
                            //}
                            //else if (i == 2)
                            //{
                            //    tag2 = "B3";
                            //}
                            else
                            {
                                tag2 = "M";
                            }
                        }

                        if (tag2num.ContainsKey(tag2) == false)
                        {
                            tag2num.Add(tag2, 1);
                        }
                        else
                        {
                            tag2num[tag2]++;
                        }

                        if (tag2.Trim().Length == 0)
                        {
                            Console.WriteLine("ERROR: {0}", strLine);
                        }

                        sw.WriteLine("{0}\t{1}", term[i], tag2);
                    }
                }

               
                sw.WriteLine();
            }

            foreach (KeyValuePair<string, int> pair in tag2num)
            {
                Console.WriteLine("{0}\t{1}", pair.Key, pair.Value);
            }
            sr.Close();
            sw.Close();
        }
    }
}
